# encoding:utf-8

from multiprocessing.managers import BaseManager

from SpiderNode.HtmlDownloader import HtmlDownloader
from SpiderNode.HtmlParser import HtmlParser

class SpiderWork(object):
    def __init__(self):
        BaseManager.register('get_task_queue')
        BaseManager.register('get_result_queue')

        server_addr = '127.0.0.1'
        print('Connect to server %s...' % server_addr)
        self.m = BaseManager(address=(server_addr, 8111), authkey='baike')
        self.m.connect()
        self.task = self.m.get_task_queue()
        self.result = self.m.get_result_queue()

        self.downloader = HtmlDownloader()
        self.parser = HtmlParser()
        print 'init finish'


    def crawl(self):
        while(True):
            try:
                if not self.task.empty():
                    url = self.task.get()

                    if url == 'end':
                        print 'ControlNode notice stop working...'
                        self.result.put({'new_urls':'end', 'data':'end'})
                        return

                    print 'Now Parsing : %s ' % url.encode('utf-8')
                    content = self.downloader.download(url)
                    new_urls, data = self.parser.parse(url, content)
                    self.result.put({"new_urls":new_urls, "data":data})
            except EOFError, e:
                print 'Connect work node failed...'
                return

            except Exception, e:
                print e
                print 'Crawl failed...'


if __name__ == '__main__':
    spider = SpiderWork()
    spider.crawl()



